{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MountainCar with Twin Delayed DDPG (TD3)\n",
    "\n",
    "### Paper: https://arxiv.org/abs/1802.09477\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Start the Environment and Agent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device:  cpu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rafaels/ml-agents/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import gym\n",
    "## import os\n",
    "\n",
    "import time\n",
    "from TwinDelayed import Actor, Critic, ReplayBuffer, TD3\n",
    "from collections import deque\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_dim:  2 , action_dim:  1\n",
      "max_action:  1.0 , threshold:  90.0\n"
     ]
    }
   ],
   "source": [
    "start_timestep=1e4\n",
    "\n",
    "std_noise=0.03\n",
    "\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "seed = 12345\n",
    "env.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "state = env.reset()\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0] \n",
    "max_action = float(env.action_space.high[0])\n",
    "threshold = env.spec.reward_threshold\n",
    "\n",
    "print('start_dim: ', state_dim, ', action_dim: ', action_dim)\n",
    "print('max_action: ', max_action, ', threshold: ', threshold)\n",
    "\n",
    "agent = TD3(state_dim, action_dim, max_action)\n",
    "\n",
    "dir_chk = 'dir_mcar_td3'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Traning the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save(agent = agent, filename='checkpnt, directory = 'dir_chkpoint')     \n",
    "def save(agent, filename, directory):\n",
    "    torch.save(agent.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))\n",
    "    torch.save(agent.actor_target.state_dict(), '%s/%s_actor_t.pth' % (directory, filename))\n",
    "    torch.save(agent.critic_target.state_dict(), '%s/%s_critic_t.pth' % (directory, filename))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Low in action space:  [-1.] , High:  [1.] , Action_dim:  1\n",
      "Ep. 10, Timestep 9990,  Ep.Timesteps 999, Score: -32.86, Avg.Score: -33.144, Max.Score: -31.67,  Time: 00:02:13 \n",
      "Ep. 20, Timestep 19980,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -21.510, Max.Score: -0.09,  Time: 00:04:48 \n",
      "Ep. 30, Timestep 29970,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -14.375, Max.Score: -0.09,  Time: 00:07:24 \n",
      "Ep. 40, Timestep 39960,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -10.806, Max.Score: -0.09,  Time: 00:10:07 \n",
      "Ep. 50, Timestep 49950,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -8.666, Max.Score: -0.08,  Time: 00:13:06 \n",
      "Ep. 60, Timestep 59940,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -7.241, Max.Score: -0.08,  Time: 00:16:27 \n",
      "Ep. 70, Timestep 69930,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -6.221, Max.Score: -0.08,  Time: 00:20:37 \n",
      "Ep. 80, Timestep 79920,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -5.456, Max.Score: -0.08,  Time: 00:25:48 \n",
      "Ep. 90, Timestep 89910,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -4.861, Max.Score: -0.08,  Time: 00:31:11 \n",
      "Ep. 100, Timestep 99900,  Ep.Timesteps 999, Score: -0.13, Avg.Score: -4.385, Max.Score: -0.08,  Time: 00:36:40 \n",
      "Ep. 110, Timestep 109890,  Ep.Timesteps 999, Score: -0.16, Avg.Score: -1.082, Max.Score: -0.08,  Time: 00:42:13 \n",
      "Ep. 120, Timestep 119880,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.106, Max.Score: -0.08,  Time: 00:47:51 \n",
      "Ep. 130, Timestep 129870,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -0.107, Max.Score: -0.08,  Time: 00:53:36 \n",
      "Ep. 140, Timestep 139860,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.108, Max.Score: -0.08,  Time: 00:59:29 \n",
      "Ep. 150, Timestep 149850,  Ep.Timesteps 999, Score: -0.13, Avg.Score: -0.107, Max.Score: -0.08,  Time: 01:05:30 \n",
      "Ep. 160, Timestep 159840,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.107, Max.Score: -0.08,  Time: 01:11:31 \n",
      "Ep. 170, Timestep 169830,  Ep.Timesteps 999, Score: -0.12, Avg.Score: -0.108, Max.Score: -0.08,  Time: 01:17:34 \n",
      "Ep. 180, Timestep 179820,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.109, Max.Score: -0.08,  Time: 01:23:35 \n",
      "Ep. 190, Timestep 189810,  Ep.Timesteps 999, Score: -0.19, Avg.Score: -0.110, Max.Score: -0.08,  Time: 01:29:38 \n",
      "Ep. 200, Timestep 199800,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.111, Max.Score: -0.08,  Time: 01:35:43 \n",
      "Ep. 210, Timestep 209790,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.109, Max.Score: -0.08,  Time: 01:41:47 \n",
      "Ep. 220, Timestep 219780,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.107, Max.Score: -0.08,  Time: 01:47:54 \n",
      "Ep. 230, Timestep 229770,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.106, Max.Score: -0.08,  Time: 01:54:00 \n",
      "Ep. 240, Timestep 239760,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.106, Max.Score: -0.08,  Time: 02:00:11 \n",
      "Ep. 250, Timestep 249750,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.105, Max.Score: -0.08,  Time: 02:06:26 \n",
      "Ep. 260, Timestep 259740,  Ep.Timesteps 999, Score: -0.15, Avg.Score: -0.105, Max.Score: -0.08,  Time: 02:12:43 \n",
      "Ep. 270, Timestep 269730,  Ep.Timesteps 999, Score: -0.16, Avg.Score: -0.107, Max.Score: -0.08,  Time: 02:19:03 \n",
      "Ep. 280, Timestep 279720,  Ep.Timesteps 999, Score: -0.12, Avg.Score: -0.110, Max.Score: -0.08,  Time: 02:25:23 \n",
      "Ep. 290, Timestep 289710,  Ep.Timesteps 999, Score: -0.19, Avg.Score: -0.111, Max.Score: -0.08,  Time: 02:31:43 \n",
      "Ep. 300, Timestep 299700,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.112, Max.Score: -0.08,  Time: 02:38:02 \n",
      "Ep. 310, Timestep 309690,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.114, Max.Score: -0.08,  Time: 02:44:23 \n",
      "Ep. 320, Timestep 319680,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.115, Max.Score: -0.09,  Time: 02:50:45 \n",
      "Ep. 330, Timestep 329670,  Ep.Timesteps 999, Score: -0.29, Avg.Score: -0.117, Max.Score: -0.09,  Time: 02:57:05 \n",
      "Ep. 340, Timestep 339660,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.121, Max.Score: -0.09,  Time: 03:03:25 \n",
      "Ep. 350, Timestep 349650,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.121, Max.Score: -0.08,  Time: 03:09:46 \n",
      "Ep. 360, Timestep 359640,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.120, Max.Score: -0.08,  Time: 03:16:09 \n",
      "Ep. 370, Timestep 369630,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.117, Max.Score: -0.08,  Time: 03:22:30 \n",
      "Ep. 380, Timestep 379620,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.115, Max.Score: -0.08,  Time: 03:28:51 \n",
      "Ep. 390, Timestep 389610,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.113, Max.Score: -0.08,  Time: 03:35:12 \n",
      "Ep. 400, Timestep 399600,  Ep.Timesteps 999, Score: -0.18, Avg.Score: -0.114, Max.Score: -0.08,  Time: 03:41:32 \n",
      "Ep. 410, Timestep 409590,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.113, Max.Score: -0.08,  Time: 03:47:53 \n",
      "Ep. 420, Timestep 419580,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.112, Max.Score: -0.08,  Time: 03:54:13 \n",
      "Ep. 430, Timestep 429570,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.111, Max.Score: -0.08,  Time: 04:00:35 \n",
      "Ep. 440, Timestep 439560,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.107, Max.Score: -0.08,  Time: 04:06:56 \n",
      "Ep. 450, Timestep 449550,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.107, Max.Score: -0.09,  Time: 04:13:18 \n",
      "Ep. 460, Timestep 459540,  Ep.Timesteps 999, Score: -0.18, Avg.Score: -0.113, Max.Score: -0.08,  Time: 04:19:39 \n",
      "Ep. 470, Timestep 469530,  Ep.Timesteps 999, Score: -0.14, Avg.Score: -0.116, Max.Score: -0.08,  Time: 04:26:01 \n",
      "Ep. 480, Timestep 479520,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.115, Max.Score: -0.08,  Time: 04:32:22 \n",
      "Ep. 490, Timestep 489510,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.116, Max.Score: -0.08,  Time: 04:38:44 \n",
      "Ep. 500, Timestep 499500,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.114, Max.Score: -0.08,  Time: 04:45:06 \n",
      "Ep. 510, Timestep 509490,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.114, Max.Score: -0.08,  Time: 04:51:28 \n",
      "Ep. 520, Timestep 519480,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -0.116, Max.Score: -0.08,  Time: 04:57:50 \n",
      "Ep. 530, Timestep 529470,  Ep.Timesteps 999, Score: -0.19, Avg.Score: -0.117, Max.Score: -0.08,  Time: 05:04:17 \n",
      "Ep. 540, Timestep 539460,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.116, Max.Score: -0.08,  Time: 05:10:41 \n",
      "Ep. 550, Timestep 549450,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.119, Max.Score: -0.08,  Time: 05:17:05 \n",
      "Ep. 560, Timestep 559440,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.113, Max.Score: -0.08,  Time: 05:23:27 \n",
      "Ep. 570, Timestep 569430,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -0.110, Max.Score: -0.08,  Time: 05:29:51 \n",
      "Ep. 580, Timestep 579420,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.111, Max.Score: -0.08,  Time: 05:36:17 \n",
      "Ep. 590, Timestep 589410,  Ep.Timesteps 999, Score: -0.11, Avg.Score: -0.110, Max.Score: -0.08,  Time: 05:42:41 \n",
      "Ep. 600, Timestep 599400,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.110, Max.Score: -0.08,  Time: 05:49:05 \n",
      "Ep. 610, Timestep 609390,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.109, Max.Score: -0.08,  Time: 05:55:30 \n",
      "Ep. 620, Timestep 619380,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.109, Max.Score: -0.08,  Time: 06:01:56 \n",
      "Ep. 630, Timestep 629370,  Ep.Timesteps 999, Score: -0.09, Avg.Score: -0.108, Max.Score: -0.08,  Time: 06:08:19 \n",
      "Ep. 640, Timestep 639360,  Ep.Timesteps 999, Score: -0.12, Avg.Score: -0.111, Max.Score: -0.09,  Time: 06:14:43 \n",
      "Ep. 650, Timestep 649350,  Ep.Timesteps 999, Score: -0.17, Avg.Score: -0.110, Max.Score: -0.08,  Time: 06:21:07 \n",
      "Ep. 660, Timestep 659340,  Ep.Timesteps 999, Score: -0.15, Avg.Score: -0.116, Max.Score: -0.08,  Time: 06:27:31 \n",
      "Ep. 670, Timestep 669330,  Ep.Timesteps 999, Score: -0.17, Avg.Score: -0.123, Max.Score: -0.08,  Time: 06:33:55 \n",
      "Ep. 680, Timestep 679320,  Ep.Timesteps 999, Score: -0.15, Avg.Score: -0.129, Max.Score: -0.08,  Time: 06:40:18 \n",
      "Ep. 690, Timestep 689310,  Ep.Timesteps 999, Score: -0.12, Avg.Score: -0.134, Max.Score: -0.08,  Time: 06:46:48 \n",
      "Ep. 700, Timestep 699300,  Ep.Timesteps 999, Score: -0.16, Avg.Score: -0.138, Max.Score: -0.08,  Time: 06:53:17 \n",
      "Ep. 710, Timestep 709290,  Ep.Timesteps 999, Score: -0.25, Avg.Score: -0.146, Max.Score: -0.08,  Time: 06:59:40 \n",
      "Ep. 720, Timestep 719280,  Ep.Timesteps 999, Score: -0.16, Avg.Score: -0.152, Max.Score: -0.08,  Time: 07:06:04 \n",
      "Ep. 730, Timestep 729270,  Ep.Timesteps 999, Score: -0.25, Avg.Score: -0.159, Max.Score: -0.08,  Time: 07:12:26 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 740, Timestep 739260,  Ep.Timesteps 999, Score: -0.19, Avg.Score: -0.168, Max.Score: -0.08,  Time: 07:18:50 \n",
      "Ep. 750, Timestep 749250,  Ep.Timesteps 999, Score: -0.17, Avg.Score: -0.178, Max.Score: -0.09,  Time: 07:25:12 \n",
      "Ep. 760, Timestep 759240,  Ep.Timesteps 999, Score: -0.23, Avg.Score: -0.181, Max.Score: -0.09,  Time: 07:31:35 \n",
      "Ep. 770, Timestep 769230,  Ep.Timesteps 999, Score: -0.24, Avg.Score: -0.185, Max.Score: -0.09,  Time: 07:37:57 \n",
      "Ep. 780, Timestep 779220,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.187, Max.Score: -0.09,  Time: 07:44:19 \n",
      "Ep. 790, Timestep 789210,  Ep.Timesteps 999, Score: -0.24, Avg.Score: -0.189, Max.Score: -0.09,  Time: 07:50:42 \n",
      "Ep. 800, Timestep 799200,  Ep.Timesteps 999, Score: -0.20, Avg.Score: -0.193, Max.Score: -0.09,  Time: 07:57:04 \n",
      "Ep. 810, Timestep 809190,  Ep.Timesteps 999, Score: -0.19, Avg.Score: -0.190, Max.Score: -0.10,  Time: 08:03:27 \n",
      "Ep. 820, Timestep 819180,  Ep.Timesteps 999, Score: -0.18, Avg.Score: -0.189, Max.Score: -0.10,  Time: 08:09:50 \n",
      "Ep. 830, Timestep 829170,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.189, Max.Score: -0.10,  Time: 08:16:14 \n",
      "Ep. 840, Timestep 839160,  Ep.Timesteps 999, Score: -0.15, Avg.Score: -0.183, Max.Score: -0.10,  Time: 08:22:38 \n",
      "Ep. 850, Timestep 849150,  Ep.Timesteps 999, Score: -0.18, Avg.Score: -0.180, Max.Score: -0.10,  Time: 08:29:00 \n",
      "Ep. 860, Timestep 859140,  Ep.Timesteps 999, Score: -0.21, Avg.Score: -0.182, Max.Score: -0.10,  Time: 08:35:22 \n",
      "Ep. 870, Timestep 869130,  Ep.Timesteps 999, Score: -0.13, Avg.Score: -0.184, Max.Score: -0.10,  Time: 08:41:45 \n",
      "Ep. 880, Timestep 879120,  Ep.Timesteps 999, Score: -0.27, Avg.Score: -0.186, Max.Score: -0.10,  Time: 08:48:07 \n",
      "Ep. 890, Timestep 889110,  Ep.Timesteps 999, Score: -0.10, Avg.Score: -0.188, Max.Score: -0.10,  Time: 08:54:30 \n",
      "Ep. 900, Timestep 899100,  Ep.Timesteps 999, Score: -0.35, Avg.Score: -0.189, Max.Score: -0.08,  Time: 09:00:52 \n",
      "Ep. 910, Timestep 909090,  Ep.Timesteps 999, Score: -0.35, Avg.Score: -0.204, Max.Score: -0.08,  Time: 09:07:14 \n",
      "Ep. 920, Timestep 918555,  Ep.Timesteps 999, Score: -0.41, Avg.Score: 2.759, Max.Score: 99.07,  Time: 09:13:13 \n",
      "Ep. 930, Timestep 928545,  Ep.Timesteps 999, Score: -0.10, Avg.Score: 2.742, Max.Score: 99.07,  Time: 09:19:25 \n",
      "Ep. 940, Timestep 938535,  Ep.Timesteps 999, Score: -2.98, Avg.Score: 2.686, Max.Score: 99.07,  Time: 09:25:35 \n",
      "Ep. 950, Timestep 948525,  Ep.Timesteps 999, Score: -0.37, Avg.Score: 2.570, Max.Score: 99.07,  Time: 09:31:43 \n",
      "Ep. 960, Timestep 958515,  Ep.Timesteps 999, Score: -0.26, Avg.Score: 2.554, Max.Score: 99.07,  Time: 09:37:51 \n",
      "Ep. 970, Timestep 968505,  Ep.Timesteps 999, Score: -0.77, Avg.Score: 2.528, Max.Score: 99.07,  Time: 09:44:00 \n",
      "Ep. 980, Timestep 978495,  Ep.Timesteps 999, Score: -0.14, Avg.Score: 2.506, Max.Score: 99.07,  Time: 09:50:13 \n",
      "Ep. 990, Timestep 987565,  Ep.Timesteps 561, Score: 96.36, Avg.Score: 4.201, Max.Score: 99.07,  Time: 09:56:09 \n",
      "Ep. 1000, Timestep 995655,  Ep.Timesteps 303, Score: 91.34, Avg.Score: 6.488, Max.Score: 99.07,  Time: 10:01:17 \n",
      "Ep. 1010, Timestep 1000416,  Ep.Timesteps 246, Score: 93.44, Avg.Score: 14.668, Max.Score: 99.07,  Time: 10:04:14 \n",
      "Ep. 1020, Timestep 1003235,  Ep.Timesteps 157, Score: 96.26, Avg.Score: 21.054, Max.Score: 97.22,  Time: 10:06:01 \n",
      "Ep. 1030, Timestep 1004825,  Ep.Timesteps 142, Score: 91.24, Avg.Score: 30.508, Max.Score: 97.22,  Time: 10:07:00 \n",
      "Ep. 1040, Timestep 1006588,  Ep.Timesteps 202, Score: 90.12, Avg.Score: 39.671, Max.Score: 97.22,  Time: 10:08:04 \n",
      "Ep. 1050, Timestep 1008313,  Ep.Timesteps 168, Score: 88.94, Avg.Score: 48.995, Max.Score: 97.22,  Time: 10:09:09 \n",
      "Ep. 1060, Timestep 1012219,  Ep.Timesteps 82, Score: 95.83, Avg.Score: 54.175, Max.Score: 97.22,  Time: 10:11:34 \n",
      "Ep. 1070, Timestep 1013645,  Ep.Timesteps 67, Score: 94.19, Avg.Score: 63.350, Max.Score: 97.22,  Time: 10:12:27 \n",
      "Ep. 1080, Timestep 1014330,  Ep.Timesteps 72, Score: 93.80, Avg.Score: 72.792, Max.Score: 97.22,  Time: 10:12:51 \n",
      "Ep. 1090, Timestep 1015009,  Ep.Timesteps 68, Score: 93.79, Avg.Score: 80.504, Max.Score: 97.22,  Time: 10:13:16 \n",
      "Ep. 1100, Timestep 1015683,  Ep.Timesteps 67, Score: 93.73, Avg.Score: 87.611, Max.Score: 96.26,  Time: 10:13:41 \n",
      "Ep. 1110, Timestep 1016360,  Ep.Timesteps 70, Score: 93.55, Avg.Score: 88.831, Max.Score: 96.26,  Time: 10:14:05 \n",
      "Ep. 1120, Timestep 1017028,  Ep.Timesteps 66, Score: 93.75, Avg.Score: 88.867, Max.Score: 95.93,  Time: 10:14:30 \n",
      "Ep. 1130, Timestep 1017700,  Ep.Timesteps 68, Score: 93.64, Avg.Score: 88.817, Max.Score: 95.93,  Time: 10:14:55 \n",
      "Ep. 1140, Timestep 1018362,  Ep.Timesteps 65, Score: 93.77, Avg.Score: 89.098, Max.Score: 95.83,  Time: 10:15:20 \n",
      "Ep. 1150, Timestep 1019031,  Ep.Timesteps 68, Score: 93.64, Avg.Score: 89.277, Max.Score: 95.83,  Time: 10:15:44 \n",
      "Ep. 1156, Timestep 1019431,  Ep.Timesteps 66, Score: 93.68, Avg.Score: 90.408, Max.Score: 95.83,  Time: 10:15:59 \n",
      "Environment solved with Average Score:  90.40762049331501\n"
     ]
    }
   ],
   "source": [
    "# Twin Delayed Deep Deterministic (TD3) policy gradient algorithm\n",
    "def twin_ddd_train(n_episodes=50000, save_every=100, print_env=10):\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    time_start = time.time()                    # Init start time\n",
    "    replay_buf = ReplayBuffer()                 # Init ReplayBuffer\n",
    "    \n",
    "    timestep_after_last_save = 0\n",
    "    total_timesteps = 0\n",
    "    \n",
    "    low = env.action_space.low\n",
    "    high = env.action_space.high\n",
    "    \n",
    "    print('Low in action space: ', low, ', High: ', high, ', Action_dim: ', action_dim)\n",
    "            \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        \n",
    "        timestep = 0\n",
    "        total_reward = 0\n",
    "        \n",
    "        # Reset environment\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            # Select action randomly or according to policy\n",
    "            if total_timesteps < start_timestep:\n",
    "                action = env.action_space.sample()\n",
    "            else:\n",
    "                action = agent.select_action(np.array(state))\n",
    "                if std_noise != 0: \n",
    "                    shift_action = np.random.normal(0, std_noise, size=action_dim)\n",
    "                    action = (action + shift_action).clip(low, high)\n",
    "            \n",
    "            # Perform action\n",
    "            new_state, reward, done, _ = env.step(action) \n",
    "            done_bool = 0 if timestep + 1 == env._max_episode_steps else float(done)\n",
    "            total_reward += reward                          # full episode reward\n",
    "\n",
    "            # Store every timestep in replay buffer\n",
    "            replay_buf.add((state, new_state, action, reward, done_bool))\n",
    "            state = new_state\n",
    "\n",
    "            timestep += 1     \n",
    "            total_timesteps += 1\n",
    "            timestep_after_last_save += 1\n",
    "\n",
    "            if done:                                       # done ?\n",
    "                break                                      # save score\n",
    "\n",
    "        scores_deque.append(total_reward)\n",
    "        scores_array.append(total_reward)\n",
    "\n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "        \n",
    "        max_score = np.max(scores_deque)\n",
    "        \n",
    "        # train_by_episode(time_start, i_episode) \n",
    "        s = (int)(time.time() - time_start)\n",
    "        if i_episode % print_env == 0 or (len(scores_deque) == 100 and avg_score > threshold):\n",
    "            print('Ep. {}, Timestep {},  Ep.Timesteps {}, Score: {:.2f}, Avg.Score: {:.3f}, Max.Score: {:.2f},  Time: {:02}:{:02}:{:02} '\\\n",
    "                .format(i_episode, total_timesteps, timestep, \\\n",
    "                        total_reward, avg_score, max_score, s//3600, s%3600//60, s%60))     \n",
    "\n",
    "        agent.train(replay_buf, timestep)\n",
    "\n",
    "        if i_episode % save_every == 0 and i_episode > 0:\n",
    "\n",
    "            timestep_after_last_save %= save_every            \n",
    "            save(agent, 'weights', dir_chk)  \n",
    "        \n",
    "        if len(scores_deque) == 100 and avg_score >= threshold:\n",
    "            print('Environment solved with Average Score: ',  avg_score )\n",
    "            break \n",
    "\n",
    "    return scores_array, avg_scores_array\n",
    "\n",
    "scores, avg_scores = twin_ddd_train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'final', dir_chk)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  1156 , len of avg_scores:  1156\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAEGCAYAAACtn3UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxV9Z3/8dfn3hAghC0Q2SEIBIhsasSVgvtSqrXudap2nEFrraOd2tZaq21nfuPUcWxrW5c6tm7jUtTqWEUFF9wryCKgYlBQEZBNwpaQ5H5+f5yTcLORG3KTu/B+Przm3HPPPfdzcsPNO9/v95yvuTsiIiIimSKS6gJEREREWkPhRURERDKKwouIiIhkFIUXERERySgKLyIiIpJRclJdQHvq27evFxUVpboMEZGMMn/+/A3uXpjqOkSak9XhpaioiHnz5qW6DBGRjGJmq1Jdg8ieqNtIREREMorCi4iIiGQUhRcRERHJKFk95kVERNLH/Pnz98vJybkLGIf+eJbmxYAl1dXV/3TwwQd/0dQGKQ0vZnY3MB34wt3HhesKgIeBImAlcLa7bzYzA34DnALsAC5y93dSUbeIiLReTk7OXf379x9bWFi4ORKJaGI9aVIsFrP169eXrF279i7g1Ka2SXXy/TNwUoN1PwbmuPsoYE54H+BkYFR4mwHc1kE1iohIcowrLCwsV3CRPYlEIl5YWLiFoIWu6W06sJ5G3H0usKnB6tOAe8Lle4Cvx62/1wNvAr3MbEDHVCoiIkkQUXCRRIQ/J81mlFS3vDSln7uvCZfXAv3C5UHAp3HbfRauq8fMZpjZPDObt379+vatVETS1t8Wr2Hp51tSXYaItIN0DC913N2BVqV0d7/T3UvdvbSwUBeIFNlXffd/3+Grv32ViqqaVJciaeZHP/pR/5EjRx5QXFxcMmbMmJIXXnihW6prktZJx7ON1pnZAHdfE3YL1Y40Xg0MidtucLhORKRZlVUxunSKtmkf1TUxbntpBUeP2Y9xg3omqTJJhdmzZ3d79tlne7377rvLunbt6mvWrMmprKy0vd1fVVUVnTp1SmaJkoB0bHl5ErgwXL4QeCJu/QUWOAzYEte9JCLSJG9d422TZi1dy83PL+fu1z6uW/fZ5h1878EFrNmyk/mrNvOfs95nzZadAGzavot5KzextaKq0b7WbNlJZbVag1Jl9erVnQoKCqq7du3qAAMGDKguKiqqevnll/MOPPDAMaNHjy4ZP3782M2bN0d27NhhZ555ZlFxcXHJ2LFjS/7v//6vO8Bvf/vbPsccc8zIww47rPiII44YDXDdddf1Gzdu3Nji4uKSq666amAqj3FfkOpTpR8EpgF9zewz4HrgRuARM7sYWAWcHW7+NMFp0mUEp0p/u8MLFpGM40kYHjpv5WYA8jvv/sicce98lq0p5/8WfV637raXVvDzUw/g1hc+ZMO2XQBcfeJovjZhILfPXcHPppdw+H+8wOH79+GiI4s4oaQfwVUg9j1Xz1w0ZPnarXnJ3Gdx/+47bjpz4qd72ubrX/96+X/8x38MLCoqGnfUUUeVn3feeZuOPfbY7eeff/6IBx54YMXUqVN3bNq0KZKfnx/7t3/7t35mxvLly5ctWLCgyymnnDJqxYoVSwCWLl2at3jx4qX9+vWreeyxx3qUlZV1Wbx48XvuznHHHTfymWeeyT/55JO3JfP4ZLeUhhd3P6+Zh45tYlsHvtu+FYmINLZy43YAInFBY1W4rqHrn1xa7/5Nz37A3a9+zMbtuzihJDj/4I2PNvLGRxuZWlzI5OEFjCjsRszBgP16dCYnEsEMenTpxK6aGPt170xVjVNRVUPnThHwYDDgruoYW3ZW0SkaoWunKDXu9M3PZceuGrrkROmZp+6Mhnr27BlbsmTJslmzZnWfM2dO9wsvvHDEVVddtWa//farmjp16g6AgoKCGMDrr7+e/73vfe8LgAMPPLBi4MCBu959990uAFOmTCnv169fDcCsWbN6zJ07t0dJSUkJwI4dOyLvv/9+F4WX9pOOY15ERJImGeflVtcEe/G4ZpztuxLv+tm4PWiFaTh4+OXl63l5efucFTlxSC+e+O6R7bLvZGiphaQ95eTkMH369K3Tp0/fOmHChJ233357q8/uyMvLi9UuuztXXnnlmquvvnpDciuV5ii8iEhW8yT0G9WOm6nd0+df7tyr/exoEHiW/eJE3GH5uq3UxJxtldWUV1ST3zlKVY2zvbKaiBkfbdhO9845dMkNBh4bQStQbV09unRi/dZKtldWE40aPbp0oqBb7l7VmO0WLVrUORKJMH78+EqABQsWdB01alTFSy+91PPll1/Omzp16o7NmzdH8vPzY0ceeeS2+++/v+DUU0/dunjx4s5r1qzJnTBhQsVbb71Vr7vr5JNPLr/hhhsGzpgxY1PPnj1jH3/8cafc3FwfNGhQdWqOMvspvIiItCA+/zzw1iqufXxJvcdzoxFKi3rz+oqNe9zPlzvqD+CNRozOOVEOHNo7abXKnpWXl0evuOKKoeXl5dFoNOpFRUWV99xzz6rly5dvuOKKK4ZWVFREunTpEps7d+7yH/7wh19ccMEFw4qLi0ui0Sh33HHHytqBvvG+8Y1vlC9durTLIYccMgaCVpkHHnjgY4WX9qPwIiJZLZmXc3WnUXABuPfiyRy2fx8ATvnNKyxbU97k8zeF3Ue1ou01WPfpq6FyK5x+e/vsP4NNmTJlx4IFC95vuH7AgAHVixYtarR+5syZKxuuu+KKKzYC9ZLqdddd98V1113X5CSCknwKLyKS1ZJxtlHtPpo77Tovd/d1ZMb0786yNeWMKOzGivX1B/X+7sWyevejkRbCS001bFsHu7bDrm1QtQMsAtHOEO0Emz+G9cuDfqRYDWxdC5XlsORROOD0Vh+nSKZQeBERaaOcyO5LZnWKBsuH7d+HXnm5zF+1udnn7fE06S2r4Y/HwLa1iReS1xe69ISiKXDYZYk/TyTDKLyISFZLxkXq6gbsJrCr3JwgvOTlRplaXMj8VZt56ntH8afXVvLoO58l/qJPXRkEl6OvhYL9IbcbdOoaFFGzK7h17Q2DDwlaY9whR4N0Zd+g8CIi2S2p3Ub1nVM6hIL8XMb07163rrYrqGtuDjO+sj9fnTCAEYX5HDt2v8TDS6wGPnkLBh4EU3/Y9gMQyTIKLyKS1ZI5YLehIQVdufyYUfXWxcKkk5cbpUunKCMK84FgWErC1iyCyi1wuK7LKdKUdJzbSESkTZJxbZd6+6vb7+51I/fL59tHDm+0bXVsd3iJ16oTi1a+EnwtmtKKJ4nsOxReRCSrJTfH7N7ZzWdNpFvnxo3XVdXBhVdzo234eP14LvQdDd377f0+pFn33XdfLzM7eMGCBV1SXQvA2rVro4ceemhxXl7egRdccMHQ+MdeeeWVvOLi4pKhQ4eOu+iii4bEYsHP17p166JHHHHEqGHDho074ogjRq1fv75tU6eHpk6dOnLDhg1t2tdTTz3V/eijjx6ZjHqao/AiIlknPrAkY8BuU7to7jTn2qvo5jUKNgk2vdRUwao3YPhXWlGgtMZDDz1UcNBBB2279957C1JdC0BeXp7/4he/+PyGG25oNCjqsssuG3bbbbetWrly5ZKPPvqoy8yZM3sAXH/99QOmTZu2ddWqVUumTZu29Wc/+1n/ZNTy8ssvl/Xt2zftpz1XeBGRrNNe41wSacXZVhlcVDW/8152G61+B6q2w3B1GbWHLVu2RN5+++38P/3pTysff/zxuvAyffr0/R966KGetffPOOOMoj/96U+9t27dGjnllFP2HzFixAHHH3/8iAkTJoyZO3duo9mwn3jiie5jx44tKS4uLjnrrLOKdu7caQCDBg0af9VVVw0sKSkZW1xcXNJUa0+PHj1iJ5544rYuXbrE4tevWrWq07Zt2yLHHnvs9kgkwvnnn7/xr3/9a2+AWbNm9brkkks2AlxyySUbn3nmmUaXaa6uruaSSy4ZPG7cuLHFxcUlN910U18IWkZKS0tHT5s2bWRRUdG4b37zm0NraoK8MmjQoPFr1qzJKS8vj0ybNm3k6NGjS0aNGnXAH//4x957Os6ZM2f2GD58+AElJSVjZ86c2au2hvLy8shZZ51VNH78+LFjx44tuf/++3sBzJs3r8v48ePHjhkzpqS4uLjk3Xff7dyKt1EDdkUkuyXlInWtOFV6x64gvHTL3YuP17//EebeFCxn+3iXv353CF8saxQC2mS/kh18/fd7nPDxf//3f3tNmzZty4QJEyp79+5d/corr+RNmTJlx9lnn73pkUce6X3uueduqaiosNdee63HPffcs+pXv/rVfr169apZsWLF0rfffrvL4YcffkDDfe7YscMuueSS4c8999wHEyZMqDz99NOLbrrppsKf/exnXwD07du3etmyZe/deOONhTfeeGO/hx9+eFUih7Nq1apOAwYMqJtTYtiwYbvWrFnTCWDjxo05w4YNqwIYMmRI1caNGxv9wP3617/u27Nnz5olS5a8t3PnTjvkkEPGfO1rXysHePfdd7stWLBgSXFx8a6vfOUro+69997e3/72t+suSvTYY4/16N+/f9VLL71UFr5etLnj/MEPfrD+8ssvL3r++ec/OOCAAyqnT5++f+1+fvKTnww4+uijy//yl7+s3LBhQ7S0tHTsqaeeWn7rrbcWXnbZZeu+853vbKqoqLDq6tbNpKCWFxHJOvEDdpPRCtNUaGmuJeVHJ41h1H75jB/cs/72ibzQ3/8IFoWjroK8tOjRyDqPPPJIwXnnnbcZ4Iwzzth03333FQCceeaZW954443uO3futJkzZ/acPHny1vz8fH/99dfzzzvvvE0AhxxySEVxcfGOhvtctGhRl8GDB1dOmDChEuCiiy7a+Oqrr9adP//Nb35zM8DkyZN3fPrpp61qYUhEJBJp8oKHs2fP7vHII4/0GTNmTMmBBx44dvPmzTnLli3rAjB+/PjtJSUlu3Jycjj77LM3vfLKK/nxzz3ooIN2vvLKKz2+853vDJo1a1Z+nz59apo7zoULF3YZPHhw5fjx4ytrW4hq9/PSSy/1uOWWWwaMGTOm5KijjhpdWVlpZWVluYcffvj2m2++ecC1117b/8MPP8zNz89v1T9VtbyISNZpt26juD1bM3GktKiA578/tdH6PV5NF2DLZ7DhAzjh3+GIy9tUZ0ZooYWkPaxbty765ptvdv/ggw+6Xn755dTU1JiZeSwW+ywvL88PO+ywrY899liPhx9+uPe55567KVmv26VLFwfIycnx6urqhM87GzZsWFVtSwvAqlWrcmtbYvr06VO9atWqTsOGDatatWpVp4KCgkZNF+5uN9988ydnnHFGvcm2nnrqqe4Nfx4b3p8wYULlO++8s+zRRx/ted111w2aPXt2+RlnnPFlorXH1cDMmTPLJk6cWBm//qCDDqqYMmXK9scff7zn9OnTR916662rTj311K2J7jctW17MbLSZLYy7lZvZlWZ2g5mtjlt/SqprFZH0lozTpps6Vbq1cyq2uPmKF4OvI45p3Y4lYffdd1/v008/fdPnn3/+7urVq99du3bt4sGDB+969tln8wHOOeeczX/+85/7vv32291rf+Effvjh2x566KHeAPPnz++yfPnyrg33O3HixIrVq1fnLlmypDPAvffe22fKlCkJ/yJuzrBhw6ry8/Njc+bM6RaLxXjggQf6nHbaaV8CnHjiiV/ecccdfQDuuOOOPieddFKjYHH88cdvue222worKysNYPHixZ3Ly8sjEHQbvf/++7k1NTXMnDmzoGG9K1eu7NS9e/fYZZddtun73//+2oULF+Y1d5yTJk2qWL16de7SpUs7QzAgunY/Rx99dPnNN9/cr/Ysqddee60rwLJly3LHjh1b+dOf/vSLE0888cuFCxc2+r7uSVq2vLj7B8AkADOLAquBx4FvA7e4+3+lsDwRSXP1zjZqp2aYpE8IvWIOdB8A+41N8o6l1l/+8peCq6++ut5kUaeddtrm+++/v+Dkk0/edvrpp5dfcsklw48//vgva1tLrr766vVnn3120YgRIw4YMWJExciRIyt69+5d72ycvLw8v/3221eeddZZI2pqapg4ceKOH/zgB+tbU9ugQYPGb9u2LVpVVWXPPvtsr6effnr5wQcfXPH73/9+1cUXXzy8oqLCjj766PKzzjprC8DPf/7zNaeffvqIYcOG9R00aNCuxx9/fEXDfV511VUbVq5c2Xn8+PFj3d0KCgqqnn766RUA48aN237ppZcOXblyZZcjjjii/Fvf+la98DN//vyu11xzzeBIJEJOTo7/4Q9/WNXccXbt2tVvvfXWVdOnTx/ZtWvX2KGHHrpt27ZtUYAbb7zx8xkzZgwdM2ZMSSwWsyFDhlS++OKLZffff3/BI4880icnJ8cLCwurfvnLX65pzffLkn0xp2QzsxOA6939SDO7AdiWaHgpLS31efPmtWt9IpJ+KqtrGP3TWQC88sOjGVLQtnGh3/jDa7zzyZecefBgZs4PzmaddeUUxvTvkfA+5ry3jovvqf95tPLGrwYLsRq4aQSMPgW+/oc21ZoMZjbf3UuTvd9FixatnDhx4oZk77c9VVdXs2vXLsvLy/OlS5d2PuGEE4pXrFixpDbcZKKnnnqq+80339zvxRcbTHOeZhYtWtR34sSJRU09lpYtLw2cCzwYd/9yM7sAmAf8q7vXm7LVzGYAMwCGDq13rR8Rkb3SZLdR6y74v+eWmjULYedmdRmloa1bt0amTJkyuqqqytydW265ZVUmB5dskdbhxcxygVOBa8JVtwG/JPgs+SVwM/CP8c9x9zuBOyFoeemwYkUkbWRct9GKF4Kv+09L4k4lGXr37h1bsmTJe6muI5mmT5++dfr06W0ek5NKaR1egJOBd9x9HUDtVwAz+yPwVKoKE5F9x+5ZpePPNmqd+Jaae/9xMiP2izsztewFGDARuvVtQ5UZIRaLxSwSiegPS9mjWCxmQKy5x9PybKM45xHXZWRmA+IeOx1Y0uEViUhGScr0ALt3VqfVLS9x2xd278ygXuHJFRXl8NnfYcSxbS4vAyxZv359z/AXk0iTYrGYrV+/vid7+B2fti0vZtYNOB64JG71r8xsEsFHyMoGj4mIAMnvNkpG/In/bV0v+Kx8FWLV+8R4l+rq6n9au3btXWvXrh1H+v/xLKkTA5ZUV1f/U3MbpG14cfftQJ8G676VonJEJIMktbUF6hJQ/b0mqfFgxRzo1A2GTE7O/tLYwQcf/AXBOEaRNlHyFZGslswYE39piVZfpC7uCfXOVFrxAhQdBTlJv2q8SNZSeBGRrFO/2yh5V9iN14YhL7tt+hg2fQQj94nxLiJJo/AiIlmn/eY2So66RpiPNCWAyN5QeBGRrJbMWaXrz2209xepq1ssmwM9h0CfkW2qT2Rfo/AiIlknvqsoOWcbNd5JW67zAkBNNXw8F0Yc3Q4TJYlkN4UXEck6HdFt1PoBuw2WV8+HynJ1GYnsBYUXEck69VtbkjBgt67bKP4Ku21pLbHgLCOLwPCpbStOZB+k8CIiWa295jZqrUZRp+x5GHgQ5BWkohyRjKbwIiLZp32uUdembqP49BKt2Air34FRx7e1NJF9ksKLiGSd+AG2Sc0xSdpZ3icvBztTeBHZKwovIpLV2mtuo1YP2I1resn75EXI6wsDDmxbYSL7KIUXEck6yR7n4nVzG8VPD7B313mJEAvCy8hjIaKPYJG9oX85IpJ1vN5y8pJMW0JRbdSZYB8RrdgMI9VlJLK3FF5EJKu119lGe3ui9LToQhzT9V1E2kDhRUSyTjImY2x6v7uX93ZW6WmRRVT2Owi69UliZSL7FoUXEck69bqNkjFgt+5U6b2/SF3n7au5r9P/Y1JkBRVFR7e9KJF9WE6qC2iOma0EtgI1QLW7l5pZAfAwUASsBM52982pqlFE0l8yxrwkYx99y2YyMboEgIphx7Z5fyL7snRveTna3Se5e2l4/8fAHHcfBcwJ74uI1NNe41za0m3Uc/VcABbF9mfXfhOSWJXIvifdw0tDpwH3hMv3AF9PYS0ikqbqXaQuid1G8VqVXXZsotuGhfy6+huctuvfMJ0iLdIm6fwvyIHnzGy+mc0I1/Vz9zXh8lqgX2pKE5F9Ub0M05r08tFLmMd4uWZikisS2Tel7ZgX4Ch3X21m+wHPm9n78Q+6u5tZo7+HwqAzA2Do0KEdU6mIpJdkX6Su9mt8t1Fr0kvZbKpze7KoYkRS6xLZV6Vty4u7rw6/fgE8DkwG1pnZAIDw6xdNPO9Ody9199LCwsKOLFlE0kTyzzZqw07coWw25YOmEAs/cls9qaOI1JOW4cXMuplZ99pl4ARgCfAkcGG42YXAE6mpUEQyRTKvsEu96QESfMq6JbBtHVsHT01iHSL7tnTtNuoHPB5e1CkH+F93n2VmbwOPmNnFwCrg7BTWKCJpKulzGzWx34QbT8pmA4ThpSx4rppeRNokLcOLu38ENBrZ5u4bAV0gQUT2KNlnGzUl4QBSNgf6j6c6rx914aV9ShLZZ6Rlt5GISLIkJbt4vS9AggGkcit88gaMPE6BRSSJFF5EJOu0X7dRK3f80csQqw7CS1x6Ua+RSNsovIhI1ql/tlH79BslFEDKZkNudxg8uV1qENlXKbyISNaJDyzJiC61+6vfbdRCenEPxrvsPxVycutt39pJHUWkPoUXEZEEeWsGvWz4ELZ8AiOPCzZXt5FI0ii8iEjWiQ8ZSblI3d48KTxFmpE6QVIk2RReRCTLtT29eFNnG7XUelI2G/qOhl6NpylRw4tI2yi8iEhWS+Z43fixNHsMIFU7YdVrdV1GoK4ikWRSeBGRrJP8U6Ub73CPF6lb+SpUV9TrMqo3SFdBRqRNFF5EJOvUu8JuO73GHvNH2WzI6QrDjmzmuUovIm2h8CIiWS05s0q3cl9ls6HoKOjUpW6Vuo1EkkfhRUSyTnvNZxSv2TCy6WPYWAajjm92ewUZkbZReBGRrJPsK+zuPtsofsBuMwlkxZzga9xg3T1uLyKtpvAiIlktmY0w8Tmo2daTsjnQuwgK9m92P4oxIm2j8CIiWae95jNqcbfVlcFkjCOPa5Ru6ncbKb6ItIXCi4hknfrdRknYX93cRi3s7JM3oWp7oy4jUGuLSDIpvIhIVmsxcLRCrKVuo7LZEM2Foil73I+CjEjbpF14MbMhZvaimS0zs6Vm9i/h+hvMbLWZLQxvp6S6VhFJT8m/SF3DhWYG4JbNgaGHQ+f8Rg/pbCOR5MlJdQFNqAb+1d3fMbPuwHwzez587BZ3/68U1iYiGcGbXNzrvTV1tlHDAFL+OXyxFI7/ZTN7UWIRSZa0Cy/uvgZYEy5vNbP3gEGprUpEMlUyG2Fie9pZWdOnSNeq1/KiICPSJmnXbRTPzIqAA4G3wlWXm9liM7vbzHo385wZZjbPzOatX7++gyoVkXTSXnMbxfY0MWPZ89B9IOw3NrkvLiKNpG14MbN84FHgSncvB24DRgCTCFpmbm7qee5+p7uXuntpYWFhh9UrIukj2WcbNbWveqc711TDipeCiRibGdBizd4RkdZKy/BiZp0IgssD7v4YgLuvc/cad48BfwQmp7JGEckMyTjbaPeYl93q5Y/V86ByS7NdRlA/7GjArkjbpF14seBf+P8A77n7f8etHxC32enAko6uTUQyQ3udbRR/8bt6AaRsNlgU9p+W3BcWkSal3YBd4EjgW8C7ZrYwXPcT4Dwzm0TwObISuCQ15YlIuotvbWmvbqM6W1bD23fBkMnQtVezz7VmlkWk9dIuvLj7qzT9b/vpjq5FRDJfMrJLbWipN2DXDGIxuOs42Ll5j11GwfZJKEREgDTsNhIRaat2mtqo8X7XLYGtn0PxyXDopXt8bvzp0ZrbSKRtFF5EJOvEh4zkTNLocf+PUzY7+Pq1Xzd5Vd3mKLqItI3Ci4hknXpjXpKxv9qzjRoGobI50H88dO/f4j7U2CKSPAovIiIJqpddKsrh0zdbHOvSFAUZkbZReBGRrFMvZCRjbqO6r3E7W/kKxKphxLFtfwERaRWFFxHJasm4SF2tenMblc2G3HwYcmhCz9XcRiLJo/AiItKC2rEuu8e8eBBehk+FnNyE9qEr7IokT8LhxcyOMrNvh8uFZja8/coSEdl79c82SsL+GnwdYZ/Dl58EcxklSHlFJHkSCi9mdj3wI+CacFUn4P72KkpEpC3a+wq7x0beCRaKT0zezkUkYYm2vJwOnApsB3D3z4Hu7VWUiEg6aXiF3WOiC6HfOOg5OOF91BvzomYYkTZJNLzs8qCz1wHMrFv7lSQi0jb1uo2SvN8ebKfUPoBRJ7TquRqkK5I8iYaXR8zsDqCXmf0zMBv4Y/uVJSKy9+qdKZ2EfqPav90c5x+iz5NjsTZ1GSnIiLRNQhMzuvt/mdnxQDkwGviZuz/frpWJiKQN55ncaxi4YxM9O22jyqN0GnxIq/agriKR5GkxvJhZFJjt7kcDCiwikvbiW1uS0W00kk8ZG/mEtRTSk238pPpibopEW7WP+OyiICPSNi2GF3evMbOYmfV09y0dUZSISFvU7zZq+/4O98VgcGnuv7OsvDO76MRNrd2JNbkoInshoW4jYBvwrpk9T3jGEYC7X9EuVYmItNEwW8uM6N8oeecJWNUDop0gkhN8jeZC5+4Q6QSRCFgULAKRaIPlCFiEf+U+Por1Zy192UVFqg9NZJ+XaHh5LLylnJmdBPwGiAJ3ufuNKS5JRNKMO1wQfZ5zoy9QvboPrKmBWA3UVEGsKpiTqBUiBi/VTGrTVAPxg3RN/UYibZLogN17zCwXKA5XfeDuVe1XVtPC8Te/B44HPgPeNrMn3X1ZR9ciIunMOTKyhNdjB7Bt+kxOHj+g/sOxGti1DWqqwcNg4zXgsXA57qvXcPEfnuGt6mF0bUMXlPKKSPIkFF7MbBpwD7CSoLt2iJld6O5z26+0Jk0Gytz9o7Cuh4DTgHYNL/e+sZKXP1jPrpoY1TXeLh9CUa/mn778Nb1rNgGOJXMq3DTgCX7TGm5V+7SYB49FzKhxx73hRHeJH/fGfxsAABesSURBVK4Z5EQiuAfvZcyDAZ7uTe/DgJyoUVUTPFpd40QjRsSC7WvLqH1uNKyx4fGYNT/+oqnVNbGgvmjEiGCYBeuCfQWvbxb8PW8W/MRUx4KKgu+NEXMn5sH3rXZd8F9QWWV1DLPgigkRC94nY/d2u78DwQG4Q9TCZYLjiUYju/cVvraF21bHgraKaMSIRCJ1x+8E3/f4iQ49/N5Z3Q1qYpATgWgkwq6aWN1zciJBTdUxpyoGuVEjGomABce6aVslB0c+5a9VR3L7A+9w6PACyiuqWVdewYjCbpwyfgCPL1jN/n27sa2yhs6dIgzv042hBXnsqonwRfkuPtm0g007qhg7oDtzKscCsG1rZdNvYAKsmWURab1Eu41uBk5w9w8AzKwYeBA4uL0Ka8Yg4NO4+58B9aZ0NbMZwAyAoUOHtvkFP/9yJz97Yil98zszqHdXcqPt87FzQOV8pu54nlU5RVRYVyCIMNnAaCYVNMEbpYHgl1+n8JdhzJ3cSPgLtsEvvkS/W9Uxp8a97hd+EIqo+8XbuCanusrJjUYwIBqLYW5Ew1+g7kGNEdv9CzXXdu9rd2AJXy3BgZvVseAnIBIznCCEdAp36njd96OuBndygmRR96I5Fh6nO/juTo/acGyxGDWx4NiC46lNF/XfsPgwHT9JoYVPMZy8ujAVhkt8d/eIB9dIMYLHa9yJhkGjdj91rx5/THGvY+EWkfDwzHa/bzF3amKxuvILgQ8ZxDOxyQC89fGmuvo3bd/F2ys3A7D4s+AchNogWBNr/IM6d/n6PbxLe0etMCJtk2h46VQbXADcfbmZdWqnmtrE3e8E7gQoLS1tc9vD5h27APi3r4/jpHH927q7pm1bD3f+M/QcwrDvvg65uoCxSGvVxLwuhLg7NTFn7fXPQnWMpT8/kU3bdzHlVy/WbV/QLZdfnjaOxxd8xu++eRBmUPbFNj7ZuIMP1m1lTP8ePLloNU+/u7bRa506cWCr69M4F5HkSTS8zDOzu9g9GeP5wLz2KWmPVgND4u4PDte1m9o/QNvtc6emGmZ+G7avh4ufVXAR2Uu1LWEQBIWcqDH3h0eztaKKbp1z6Nqp/nVZivvl89UJA/jqhN3jYQ4Y2JMDBvasGyNTXlHVZHi56MiiVtdXvwtRQUakLRKdHuA7BONKrghvy8J1He1tYJSZDQ8HEJ8LPNkRL5z0j5qqnbB6Pjx1Jax8Bb72axh4YLJfRWSf1q9HF0buF8whG4kYd19UynmTg+7kvvmdW3z+WQcP5nffbPzvsndebnILFZFWSbTlJQf4jbv/N9Sd9dPyv/wkc/dqM7sceJbgVOm73X1pR7x2m/9SKv8cVr4Gq16DT96EDR8EZzIAHHYZTPpm24sUkT06Zkw/Dh3eh0lDenLEiL4tbm9mFPVp3Boa3YvPAzW2iCRPouFlDnAcwcXqALoCzwFHtEdRe+LuTwNPd9zrteHJG1fA0sfhvSdhzaJgXeceMORQGPs16D8+uBUMT0qtItKybp1zOOeQxAfz5zQxSH9vgogmYxRJnkTDSxd3rw0uuPs2M8trp5rSSu35GQl/7LhD2Wx48w+w4oVg3eBD4PhfwPCpQVhp5ZwoIpI6OZHG//ojTaxrkbKLSNIkGl62m9lB7v4OgJmVAjvbr6z00aoBu6vfgWevhU9eh/z+cPS1QXdQz8HtWqOItJ9opPHQwL3JLiKSPImGlyuBv5jZ5+H9AcA57VNSetpjeKmphrm/grk3QV4f+Op/w0EXBHOoiEhGa7LlRWNeRFJqj+HFzA4BPnX3t81sDHAJ8A1gFvBxB9SXci0OeancBn+5MOgqmngenPyf0KVnR5QmIh0g2kR42bsxLyKSLC2dKn0HsCtcPhz4CcHcQpsJLwSX7WqvJtrkYLuqCnjgzGBsy/Rfw+m3K7iIZJna8NKz6+6W1L1peRGR5Gmp2yjq7rXX1T4HuNPdHwUeNbOF7Vtammn4WeUOT1wGn7wBZ94N485ISVki0r5qw0ssbuqAves2UuARSZaWWl6iZlYbcI4FXoh7LNHxMhmt2W6jd+6FJY/CsdcruIhksdoxLzVx103Ymxii6CKSPC0FkAeBl81sA8HZRa8AmNlIYEs715YW6s42il+5dW1wVlHRFDjyylSUJSIdpLblpabNLS9JK0lkn7fH8OLu/25mcwjOLnrOve5PjwjwvfYuLp3Ua/J95Wao2gFf+w00cRqliGSPnPDfeHx4Mf2zF0mpFrt+3P3NJtYtb59y0lGDjqPyz2H+n+HA86HPiJRUJCIdJ9pEt9Fetbyo40gkafT3QwsadRsteABqdsFR309VSSLSgWrHvMRPFbJXF9hVdhFJGoWXBJkBsRgsvD8Y66L5iET2CbVTAYzp3333OiURkZRSeGlBvU6jT9+CzSvhwH9IUTUikgqPfudwHvznw+ruK7uIpNY+cbpzW+zuNjL48FmI5MDoU1JblIh0qIOHFdS7r7ONRFJLLS8tqLvCrhFMATDkMOjSI7VFiUhKacCuSGopvCSoU8VGWPsujDwm1aWISIppVmmR1Eqr8GJmN5nZ+2a22MweN7Ne4foiM9tpZgvD2+0dVVPtmJfuGxYHC0MOa3ZbEdk37M2l/tVtJJI8aRVegOeBce4+AVgOXBP32Ap3nxTeLu2ogmrHvORvWhxcmWrAxI56aRHJIsouIsmTVuHF3Z9z9+rw7pvA4FTWEy9/42LoOxo656e6FBERkX1aWoWXBv4ReCbu/nAzW2BmL5vZlOaeZGYzzGyemc1bv359m4vwsOOo26ZlMHBSm/cnIvsmzSotkjwdfqq0mc0G+jfx0LXu/kS4zbVANfBA+NgaYKi7bzSzg4G/mtkB7l7ecCfufidwJ0BpaWmzk0InzKEbO8nd+QX0Gdnm3YnIvknRRSR5Ojy8uPtxe3rczC4CpgPH1k4E6e6VQGW4PN/MVgDFwLz2rTZQZOuCBYUXEdlLangRSZ606jYys5OAHwKnuvuOuPWFZhYNl/cHRgEfdURNDhTZ2uCOJmIUERFJuXS7wu7vgM7A82H/8JvhmUVfAX5hZlVADLjU3Td1REHuMNzWBHcK9u+IlxSRLKQxLyLJk1bhxd2b7Jdx90eBRzu4nOC1cYoi69iV14/c3G6pKEFERETipFV4SSdLVm9h+q2vUjKgBz9hI7u6DSQ31UWJiIiIwktzvv+7B/l/ObPYtb4TR+UsZXPXE1JdkoiIiKDw0qRYTYw/5/4nA233sJqqvP1SWJGIiIjUSquzjdLFxtUfMtA2MbPmKzxbUwpAddfCFFclIiIioPDSpILBxXx+0d857l/uoptVAlCl8CIiIpIW1G3UhGjEGFg0GoA8KgCoVreRiIhIWlDLSwvqwotaXkRERNKCWl5a0M3C8NK5d4orEZFUm3XlFD7/cmeqyxDZ56nlpQVX1VzB0zWTqeo+KNWliEiKjenfg2PG9Et1GSL7PIWXFizykVxWdSUWUSOViIhIOlB4aYFpInsREZG0ovCSIM2pJiIikh4UXhKkFhgREZH0oPCSILW8iIiIpAeFl5YotIiIiKQVhRcRERHJKGkXXszsBjNbbWYLw9spcY9dY2ZlZvaBmZ3YsXV15KuJiIhIc9L14iW3uPt/xa8wsxLgXOAAYCAw28yK3b2mPQtRZhEREUkvadfysgenAQ+5e6W7fwyUAZM76sV1tpGIiEh6SNfwcrmZLTazu82sdlKhQcCncdt8Fq6rx8xmmNk8M5u3fv36NhdS212kbiMREZH0kJLwYmazzWxJE7fTgNuAEcAkYA1wc2v27e53unupu5cWFrZ9Jmi1uIiIiKSXlIx5cffjEtnOzP4IPBXeXQ0MiXt4cLiuQ6jlRUREJD2kXbeRmQ2Iu3s6sCRcfhI418w6m9lwYBTw9w6rSy0wIiIiaSEdzzb6lZlNAhxYCVwC4O5LzewRYBlQDXy3vc80ArW4iIiIpJu0Cy/u/q09PPbvwL93YDl1FGJERETSQ9p1G6UrZRcREZH0oPCSILW8iIiIpAeFlxYos4iIiKQXhZeEKcaIiIikA4WXBKnbSEREJD0ovLTAlFpERETSisJLghRhRERE0oPCS4LUAiMiIpIeFF5aoMgiIiKSXhReEqQQIyIikh4UXhKkXiMREZH0oPDSEqv9ovQiIiKSDhReWqDIIiIikl4UXhKkbiMREZH0oPAiIiIiGUXhpQW6vouIiEh6yUl1AfHM7GFgdHi3F/Clu08ysyLgPeCD8LE33f3Sjq2tI19NREREmpNW4cXdz6ldNrObgS1xD69w90kdX1VdPal6aREREYmTVuGllgVJ4WzgmNTXkuoKREREJF66jnmZAqxz9w/j1g03swVm9rKZTWnuiWY2w8zmmdm89evXJ60gZRgREZH00OEtL2Y2G+jfxEPXuvsT4fJ5wINxj60Bhrr7RjM7GPirmR3g7uUNd+LudwJ3ApSWlnry6k7WnkRERKQtOjy8uPtxe3rczHKAbwAHxz2nEqgMl+eb2QqgGJjXjqUG9bT3C4iIiEirpGO30XHA++7+We0KMys0s2i4vD8wCvioI4vS9AAiIiLpIR0H7J5L/S4jgK8AvzCzKiAGXOrumzqyKHUbiYiIpIe0Cy/uflET6x4FHu34anZTdhEREUkP6dhtlFZ0fRcREZH0ovCSKGUYERGRtKDwkiAN2BUREUkPCi8tUGQRERFJLwovLdCQFxERkfSi8JIgJ2kX6xUREZE2UHhpkZpeRERE0onCi4iIiGQUhZdEqddIREQkLSi8tEADdkVERNKLwouIiIhkFIUXERERySgKLwnSkBcREZH0oPDSAg15ERERSS8KLyIiIpJRFF4S5Oo3EhERSQspCS9mdpaZLTWzmJmVNnjsGjMrM7MPzOzEuPUnhevKzOzHHVXr948vBqBXXqeOekkRERHZg5wUve4S4BvAHfErzawEOBc4ABgIzDaz4vDh3wPHA58Bb5vZk+6+rL0LPXfyUM6dPLS9X0ZE9gG3nncgPbvqDyGRtkpJeHH39wCs8RXgTgMecvdK4GMzKwMmh4+VuftH4fMeCrdt9/AiIpIsX5s4MNUliGSFdBvzMgj4NO7+Z+G65taLiIjIPqbdWl7MbDbQv4mHrnX3J9rxdWcAMwCGDlV3j4iISLZpt/Di7sftxdNWA0Pi7g8O17GH9Q1f907gToDS0lKdIyQiIpJl0q3b6EngXDPrbGbDgVHA34G3gVFmNtzMcgkG9T6ZwjpFREQkRVIyYNfMTgduBQqBv5nZQnc/0d2XmtkjBANxq4HvuntN+JzLgWeBKHC3uy9NRe0iIiKSWuZZfPW10tJSnzdvXqrLEBHJKGY2391LW95SJDXSrdtIREREZI8UXkRERCSjZHW3kZmtB1bt5dP7AhuSWE460DFlhmw7pmw7Hsj+Yxrm7oWpLEZkT7I6vLSFmc3Ltj5fHVNmyLZjyrbjAR2TSKqp20hEREQyisKLiIiIZBSFl+bdmeoC2oGOKTNk2zFl2/GAjkkkpTTmRURERDKKWl5EREQkoyi8iIiISEZReGmCmZ1kZh+YWZmZ/TjV9STCzIaY2YtmtszMlprZv4TrC8zseTP7MPzaO1xvZvbb8BgXm9lBqT2C5plZ1MwWmNlT4f3hZvZWWPvD4WSdhBN6Phyuf8vMilJZd3PMrJeZzTSz983sPTM7PJPfJzO7KvyZW2JmD5pZl0x7j8zsbjP7wsyWxK1r9XtiZheG239oZhem4ljiamnqmG4Kf+4Wm9njZtYr7rFrwmP6wMxOjFufcZ+Hkv0UXhowsyjwe+BkoAQ4z8xKUltVQqqBf3X3EuAw4Lth3T8G5rj7KGBOeB+C4xsV3mYAt3V8yQn7F+C9uPv/Cdzi7iOBzcDF4fqLgc3h+lvC7dLRb4BZ7j4GmEhwbBn5PpnZIOAKoNTdxxFMnHoumfce/Rk4qcG6Vr0nZlYAXA8cCkwGrq8NPCnyZxof0/PAOHefACwHrgEIPyvOBQ4In/OH8I+GTP08lCyn8NLYZKDM3T9y913AQ8BpKa6pRe6+xt3fCZe3EvxCHERQ+z3hZvcAXw+XTwPu9cCbQC8zG9DBZbfIzAYDXwXuCu8bcAwwM9yk4THVHutM4Nhw+7RhZj2BrwD/A+Duu9z9SzL7fcoBuppZDpAHrCHD3iN3nwtsarC6te/JicDz7r7J3TcTBIWG4aHDNHVM7v6cu1eHd98EBofLpwEPuXulu38MlBF8Fmbk56FkP4WXxgYBn8bd/yxclzHCpvgDgbeAfu6+JnxoLdAvXM6U4/w18EMgFt7vA3wZ9wEcX3fdMYWPbwm3TyfDgfXAn8KusLvMrBsZ+j65+2rgv4BPCELLFmA+mf0e1Wrte5LW71UT/hF4JlzOlmOSfYTCS5Yxs3zgUeBKdy+Pf8yD8+Iz5tx4M5sOfOHu81NdSxLlAAcBt7n7gcB2dndHAJn1PoXdIqcRhLKBQDdS2NrQXjLpPUmEmV1L0NX8QKprEdkbCi+NrQaGxN0fHK5Le2bWiSC4PODuj4Wr19V2M4RfvwjXZ8JxHgmcamYrCZqrjyEYL9Ir7KKA+nXXHVP4eE9gY0cWnIDPgM/c/a3w/kyCMJOp79NxwMfuvt7dq4DHCN63TH6ParX2PUn39woAM7sImA6c77sv9JXRxyT7HoWXxt4GRoVnS+QSDGJ7MsU1tSgcN/A/wHvu/t9xDz0J1J71cCHwRNz6C8IzJw4DtsQ1kacFd7/G3Qe7exHB+/CCu58PvAicGW7W8Jhqj/XMcPu0+mvZ3dcCn5rZ6HDVscAyMvd9+gQ4zMzywp/B2uPJ2PcoTmvfk2eBE8ysd9gidUK4Lm2Y2UkE3bCnuvuOuIeeBM4NzwYbTjAY+e9k6Oeh7APcXbcGN+AUgpH4K4BrU11PgjUfRdCsvRhYGN5OIRhPMAf4EJgNFITbG8FZBCuAdwnOFkn5cezh+KYBT4XL+xN8sJYBfwE6h+u7hPfLwsf3T3XdzRzLJGBe+F79Feidye8T8HPgfWAJcB/QOdPeI+BBgjE7VQStYxfvzXtCMI6kLLx9Ow2PqYxgDEvtZ8TtcdtfGx7TB8DJcesz7vNQt+y/aXoAERERySjqNhIREZGMovAiIiIiGUXhRURERDKKwouIiIhkFIUXERERySgKLyJxzKzGzBbG3fY4i66ZXWpmFyThdVeaWd+27ifcVzczmx0uvxp3sTgRkaygDzWR+na6+6REN3b329uzmL10OPBGeKG07b57jiERkayglheRBIQtI78ys3fN7O9mNjJcf4OZ/SBcvsLMlpnZYjN7KFxXYGZ/Dde9aWYTwvV9zOw5M1tqZncRXPis9rX+IXyNhWZ2h5lFw9ufzWxJWMNVTdQ4wswWAvcD3ySYIHFiuJ/92v2bJCLSQRReROrr2qDb6Jy4x7a4+3jgdwSzXTf0Y+BAd58AXBqu+zmwIFz3E+DecP31wKvufgDwODAUwMzGAucAR4YtQDXA+QRX5R3k7uPCGv7U8MXdfUX4nPnAZOAe4GJ3n+TuXzTcXkQkU6nbSKS+PXUbPRj39ZYmHl8MPGBmfyW47D8E0zacAeDuL4QtLj2ArwDfCNf/zcw2h9sfCxwMvB1MFURXggkB/w/Y38xuBf4GPLeHY9jP3TeGrTz/09IBi4hkGrW8iCTOm1mu9VWCOW8OIggfe/PHgQH3hK0lk9x9tLvf4O6bgYnASwStOnc1eqLZ7Wa2hGAivYXAScBTTXUxiYhkMoUXkcSdE/f1jfgHzCwCDHH3F4EfAT2BfOAVgm4fzGwasMHdy4G5BONSMLOTCSZnhGAiwDNrx6iEY2aGhWciRdz9UeCnBAGpHne/lKCb6pfA14G/hQGoqVYiEZGMpW4jkfq6hq0WtWa5e+3p0r3NbDFQCZzX4HlR4H4z60nQevJbd//SzG4A7g6ftwO4MNz+58CDZrYUeB34BMDdl5nZT4HnwkBUBXwX2An8KVwHcE0z9U8lGFczBXi59YcvIpL+NKu0SALMbCVQ6u4bUl2LiMi+Tt1GIiIiklHU8iIiIiIZRS0vIiIiklEUXkRERCSjKLyIiIhIRlF4ERERkYyi8CIiIiIZ5f8D3day1CPozxwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "import os\n",
    "\n",
    "def play(env, agent, n_episodes):\n",
    "    state = env.reset()\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()        \n",
    "        score = 0\n",
    "        \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            action = agent.select_action(np.array(state))\n",
    "            env.render()\n",
    "            time.sleep(0.01)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "\n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)\n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60))  \n",
    "\n",
    "play(env=env, agent=agent, n_episodes=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
